<!doctype html>



  


<html class="theme-next pisces use-motion">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  
  
  <link href="/vendors/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  

  

  

  

  
    

    
  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Microsoft YaHei Mono:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/vendors/font-awesome/css/font-awesome.min.css?v=4.4.0" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.0.1" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="Python,多线程," />





  <link rel="alternate" href="/atom.xml" title="岁月如歌" type="application/atom+xml" />




  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.0.1" />






<meta name="description" content="今天来搜一搜「他」：百度图片搜索结果。
下载简单页面查看网页源码，发现同一张图片有四种网址：">
<meta property="og:type" content="article">
<meta property="og:title" content="Python 3 多线程下载百度图片搜索结果">
<meta property="og:url" content="http://lovenight.github.io/2015/11/15/Python-3-多线程下载百度图片搜索结果/index.html">
<meta property="og:site_name" content="岁月如歌">
<meta property="og:description" content="今天来搜一搜「他」：百度图片搜索结果。
下载简单页面查看网页源码，发现同一张图片有四种网址：">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%BB%96.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%B8%8B%E8%BD%BD%E7%BB%93%E6%9E%9C1.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CF12.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CRequestURL.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E6%96%87%E6%9C%AC%E5%AF%B9%E6%AF%94%E7%BB%93%E6%9E%9C.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%B8%AD%E6%96%87%E7%BC%96%E7%A0%81%E8%A7%A3%E7%A0%81.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CResponse.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CJSON%20Handle.png">
<meta property="og:image" content="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E5%8D%95%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%BB%93%E6%9E%9C.png">
<meta property="og:updated_time" content="2015-11-16T06:45:57.578Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Python 3 多线程下载百度图片搜索结果">
<meta name="twitter:description" content="今天来搜一搜「他」：百度图片搜索结果。
下载简单页面查看网页源码，发现同一张图片有四种网址：">



<script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: 14198272,
      author: 'Author'
    }
  };
</script>




  <link rel="canonical" href="http://lovenight.github.io/2015/11/15/Python-3-多线程下载百度图片搜索结果/"/>

  <title> Python 3 多线程下载百度图片搜索结果 | 岁月如歌 </title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  





  <script type="text/javascript">
    (function() {
      var hm = document.createElement("script");
      hm.src = "//tajs.qq.com/stats?sId=58588376";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>






  
  
    
  

  <div class="container one-collumn sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">岁月如歌</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle"></p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-android">
          <a href="/Android" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-android"></i> <br />
            
            Android
          </a>
        </li>
      
        
        <li class="menu-item menu-item-python">
          <a href="/Python" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-rocket"></i> <br />
            
            Python
          </a>
        </li>
      
        
        <li class="menu-item menu-item-notes">
          <a href="/Notes" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-book"></i> <br />
            
            Notes
          </a>
        </li>
      
        
        <li class="menu-item menu-item-links">
          <a href="/Links" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-link"></i> <br />
            
            Links
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/About" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            搜索
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>


    </div>
  
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  
  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                Python 3 多线程下载百度图片搜索结果
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            <span class="post-meta-item-icon">
              <i class="fa fa-calendar-o"></i>
            </span>
            <span class="post-meta-item-text">发表于</span>
            <time itemprop="dateCreated" datetime="2015-11-15T18:58:36+08:00" content="2015-11-15">
              2015-11-15
            </time>
          </span>

          
            <span id="busuanzi_container_page_pv">&nbsp;&nbsp;|&nbsp;&nbsp;阅读量 <span id="busuanzi_value_page_pv"></span> 次</span>
          


          
            <span class="post-category" >
              &nbsp; | &nbsp;
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/编程/" itemprop="url" rel="index">
                    <span itemprop="name">编程</span>
                  </a>
                </span>

                
                
                  ， 
                

              
                <span itemprop="about" itemscope itemtype="https://schema.org/Thing">
                  <a href="/categories/编程/Python/" itemprop="url" rel="index">
                    <span itemprop="name">Python</span>
                  </a>
                </span>

                
                

              
            </span>
          

          
            
          

          

          
          

          
        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        <p>今天来搜一搜「他」：<a href="http://www.baidu.com/s?wd=%E9%95%BF%E8%80%85%E8%9B%A4&amp;tn=87048150_dg&amp;ie=utf8" target="_blank" rel="external">百度图片搜索结果</a>。<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%BB%96.png" alt="搜索结果"></p>
<h2 id="下载简单页面">下载简单页面</h2><p>查看网页源码，发现同一张图片有四种网址：</p>
<a id="more"></a>
<pre><code>"thumbURL": "http://img1.imgtn.bdimg.com/it/u=<span class="number">757023778</span>,<span class="number">2840825931</span>&amp;fm=21&amp;gp=0.jpg",
"middleURL": "http://img1.imgtn.bdimg.com/it/u=<span class="number">757023778</span>,<span class="number">2840825931</span>&amp;fm=21&amp;gp=0.jpg",
"hoverURL": "http://img1.imgtn.bdimg.com/it/u=<span class="number">757023778</span>,<span class="number">2840825931</span>&amp;fm=23&amp;gp=0.jpg",
"objURL": "http://imgsrc.baidu.com/forum/w=580/sign=b3bcc<span class="number">2f88a5494</span>ee<span class="number">87220f111</span>df<span class="number">4e0e1/78</span>fed309b3de<span class="number">9c82913</span>abac<span class="number">86a81800a18</span>d84344.jpg",
</code></pre><p>经测试，前三种都有反爬虫措施，用浏览器可以打开，但是刷新一次就403 Forbidden。用爬虫获取不到图片。</p>
<p>第四种objURL是指图片的源网址，获取该网址会出现三种情况：</p>
<ol>
<li>正常。继续下载</li>
<li>403 Forbidden。用continue跳过。</li>
<li>出现异常。用try except处理。</li>
</ol>
<p>代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"></span><br><span class="line">url = <span class="string">r'http://image.baidu.com/search/index?tn=baiduimage&amp;ipn=r&amp;ct=201326592&amp;cl=2&amp;lm=-1&amp;st=-1&amp;fm=detail&amp;fr=&amp;sf=1&amp;fmq=1447473655189_R&amp;pv=&amp;ic=0&amp;nc=1&amp;z=&amp;se=&amp;showtab=0&amp;fb=0&amp;width=&amp;height=&amp;face=0&amp;istype=2&amp;ie=utf-8&amp;word=%E9%95%BF%E8%80%85%E8%9B%A4'</span></span><br><span class="line">dirpath = <span class="string">r'F:\img'</span></span><br><span class="line"></span><br><span class="line">html = requests.get(url).text</span><br><span class="line">urls = re.findall(<span class="string">r'"objURL":"(.*?)"'</span>, html)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> os.path.isdir(dirpath):</span><br><span class="line">    os.mkdir(dirpath)</span><br><span class="line"></span><br><span class="line">index = <span class="number">1</span></span><br><span class="line"><span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">    print(<span class="string">"Downloading:"</span>, url)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        res = requests.get(url)</span><br><span class="line">        <span class="keyword">if</span> str(res.status_code)[<span class="number">0</span>] == <span class="string">"4"</span>:</span><br><span class="line">            print(<span class="string">"未下载成功："</span>, url)</span><br><span class="line">            <span class="keyword">continue</span></span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">        print(<span class="string">"未下载成功："</span>, url)</span><br><span class="line">    filename = os.path.join(dirpath, str(index) + <span class="string">".jpg"</span>)</span><br><span class="line">    <span class="keyword">with</span> open(filename, <span class="string">'wb'</span>) <span class="keyword">as</span> f:</span><br><span class="line">        f.write(res.content)</span><br><span class="line">        index += <span class="number">1</span></span><br><span class="line"></span><br><span class="line">print(<span class="string">"下载结束，一共 %s 张图片"</span> % index)</span><br></pre></td></tr></table></figure>
<p><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%B8%8B%E8%BD%BD%E7%BB%93%E6%9E%9C1.png" alt="下载结果"></p>
<p>Exciting！</p>
<h2 id="加载更多图片">加载更多图片</h2><p>但是上面的代码还有不足，当我们在网页中下拉时，百度会继续加载更多图片。需要再完善一下代码。</p>
<p>打开Chrome，按F12，切换到Network标签，然后将网页向下拉。这时浏览器地址栏的网址并没有改变，而网页中的图片却一张张增加，说明网页在后台与服务器交互数据。警察蜀黍，就是这家伙：<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CF12.png" alt="Chrome 抓包"></p>
<p>xhr全称XMLHttpRequest，详细介绍见百度：<a href="http://baike.baidu.com/link?url=59cVUdY8JuhiQ1NBfELSH93H_HFlTg9YoC5cqnjuyIvVHcyqx92fVNLTUWfvq8lesK5-h9HXgmbVuQseom0qu_" target="_blank" rel="external">XMLHTTPRequest_百度百科</a></p>
<p>点开看看：<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CRequestURL.png" alt="Request URL"></p>
<p>这么长一串网址，没有头绪。下拉网页，再抓一个xhr，对比一下它们的Request URL，使用在线文字对比工具：<a href="http://home.putclub.com/tools/find/" target="_blank" rel="external">文本比较</a><br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E6%96%87%E6%9C%AC%E5%AF%B9%E6%AF%94%E7%BB%93%E6%9E%9C.png" alt="文本对比结果"></p>
<p>URL末尾有三处变化，最后一项看上去是时间戳，经过测试，直接把它删了也没事。</p>
<p>那么只需要研究<code>pn</code>和<code>gsm</code>值。继续下拉，到底的时候点<code>加载更多图片</code>，多抓几个对比一下URL的末尾部分：</p>
<pre><code>pn=<span class="number">120</span>&amp;rn=<span class="number">60</span>&amp;gsm=<span class="number">78</span>
pn=<span class="number">180</span>&amp;rn=<span class="number">60</span>&amp;gsm=b4
pn=<span class="number">240</span>&amp;rn=<span class="number">60</span>&amp;gsm=f0
pn=<span class="number">300</span>&amp;rn=<span class="number">60</span>&amp;gsm=<span class="number">12</span>c
pn=<span class="number">360</span>&amp;rn=<span class="number">60</span>&amp;gsm=<span class="number">168</span>
</code></pre><p><code>pn</code>是一个60为步长的等差数列。<code>gsm</code>看上去是16进制，转换成十进制，发现它就是<code>pn</code>值，试了也可以删掉。</p>
<p>经测试，<code>rn</code>是步长值，最大只能取60，填入大于60的数，仍然以60为步长。如果删了<code>rn</code>，则步长值变为30。<code>pn</code>是图片编号，从0开始。</p>
<p>现在已经删了时间戳和<code>gsm</code>两项了，能不能让网址再短一点？继续观察，注意到：</p>
<pre><code>&amp;se=&amp;tab=&amp;<span class="variable">width</span>=&amp;<span class="variable">height</span>=
</code></pre><p>这几项没指定值，肯定没用，把没值的都删了。</p>
<p>再看看这两项：</p>
<pre><code>queryWord=<span class="preprocessor">%</span>E<span class="number">9</span><span class="preprocessor">%</span><span class="number">95</span><span class="preprocessor">%</span>BF<span class="preprocessor">%</span>E<span class="number">8</span><span class="preprocessor">%</span><span class="number">80</span><span class="preprocessor">%</span><span class="number">85</span><span class="preprocessor">%</span>E<span class="number">8</span><span class="preprocessor">%</span><span class="number">9</span>B<span class="preprocessor">%</span>A<span class="number">4</span>
word=<span class="preprocessor">%</span>E<span class="number">9</span><span class="preprocessor">%</span><span class="number">95</span><span class="preprocessor">%</span>BF<span class="preprocessor">%</span>E<span class="number">8</span><span class="preprocessor">%</span><span class="number">80</span><span class="preprocessor">%</span><span class="number">85</span><span class="preprocessor">%</span>E<span class="number">8</span><span class="preprocessor">%</span><span class="number">9</span>B<span class="preprocessor">%</span>A<span class="number">4</span>
</code></pre><p>这就是我们本次搜索的关键词。网址中的中文会被编码成UTF-8，每个中文3个字节，每个字节前加上%号。编码和解码方法如下：<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E4%B8%AD%E6%96%87%E7%BC%96%E7%A0%81%E8%A7%A3%E7%A0%81.png" alt="中文编码解码"></p>
<p>那么，我们可以写出指定关键词需要请求的所有网址：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> itertools</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">buildUrls</span><span class="params">(word)</span>:</span></span><br><span class="line">    word = urllib.parse.quote(word)</span><br><span class="line">    url = <span class="string">r"http://image.baidu.com/search/acjson?tn=resultjson_com&amp;ipn=rj&amp;ct=201326592&amp;fp=result&amp;queryWord=&#123;word&#125;&amp;cl=2&amp;lm=-1&amp;ie=utf-8&amp;oe=utf-8&amp;st=-1&amp;ic=0&amp;word=&#123;word&#125;&amp;face=0&amp;istype=2nc=1&amp;pn=&#123;pn&#125;&amp;rn=60"</span></span><br><span class="line">    urls = (url.format(word=word, pn=x) <span class="keyword">for</span> x <span class="keyword">in</span> itertools.count(start=<span class="number">0</span>, step=<span class="number">60</span>))</span><br><span class="line">    <span class="keyword">return</span> urls</span><br></pre></td></tr></table></figure></p>
<p>上面的代码中，itertools.count(start=0, step=60)表示一个从0开始，60为步长值的无限等差数列。<br>把这个数列的数字分别填入url作为pn值，就得到一个无限容量的url生成器，注意生成器必须用圆括号，如果写成中括号就成了列表，程序会在这一步无限执行下去。</p>
<p>下面开始解析每次获取的数据。我们点开看看，返回的是一串json数据。<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CResponse.png" alt="接口返回数据"></p>
<p>纳尼，这个objURL怎么不是HTTP开头的。试了几种方法都没成功，Google一下，找到这个：<a href="http://blog.csdn.net/hbuxiaoshe/article/details/44780653" target="_blank" rel="external">百度图片url解码</a></p>
<p>OK，既然明白了原理，我们写一个Python版的解密实现：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Author: loveNight</span></span><br><span class="line"><span class="comment"># @Date:   2015-11-14 21:11:11</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="string">"""解码百度图片搜索json中传递的url</span><br><span class="line">抓包可以获取加载更多图片时，服务器向网址传输的json。</span><br><span class="line">其中originURL是特殊的字符串</span><br><span class="line">解码前：</span><br><span class="line">ippr_z2C$qAzdH3FAzdH3Ffl_z&amp;e3Bftgwt42_z&amp;e3BvgAzdH3F4omlaAzdH3Faa8W3ZyEpymRmx3Y1p7bb&amp;mla</span><br><span class="line">解码后：</span><br><span class="line">http://s9.sinaimg.cn/mw690/001WjZyEty6R6xjYdtu88&amp;690</span><br><span class="line">使用下面两张映射表进行解码。</span><br><span class="line">"""</span></span><br><span class="line"></span><br><span class="line">str_table = &#123;</span><br><span class="line">    <span class="string">'_z2C$q'</span>: <span class="string">':'</span>,</span><br><span class="line">    <span class="string">'_z&amp;e3B'</span>: <span class="string">'.'</span>,</span><br><span class="line">    <span class="string">'AzdH3F'</span>: <span class="string">'/'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">char_table = &#123;</span><br><span class="line">    <span class="string">'w'</span>: <span class="string">'a'</span>,</span><br><span class="line">    <span class="string">'k'</span>: <span class="string">'b'</span>,</span><br><span class="line">    <span class="string">'v'</span>: <span class="string">'c'</span>,</span><br><span class="line">    <span class="string">'1'</span>: <span class="string">'d'</span>,</span><br><span class="line">    <span class="string">'j'</span>: <span class="string">'e'</span>,</span><br><span class="line">    <span class="string">'u'</span>: <span class="string">'f'</span>,</span><br><span class="line">    <span class="string">'2'</span>: <span class="string">'g'</span>,</span><br><span class="line">    <span class="string">'i'</span>: <span class="string">'h'</span>,</span><br><span class="line">    <span class="string">'t'</span>: <span class="string">'i'</span>,</span><br><span class="line">    <span class="string">'3'</span>: <span class="string">'j'</span>,</span><br><span class="line">    <span class="string">'h'</span>: <span class="string">'k'</span>,</span><br><span class="line">    <span class="string">'s'</span>: <span class="string">'l'</span>,</span><br><span class="line">    <span class="string">'4'</span>: <span class="string">'m'</span>,</span><br><span class="line">    <span class="string">'g'</span>: <span class="string">'n'</span>,</span><br><span class="line">    <span class="string">'5'</span>: <span class="string">'o'</span>,</span><br><span class="line">    <span class="string">'r'</span>: <span class="string">'p'</span>,</span><br><span class="line">    <span class="string">'q'</span>: <span class="string">'q'</span>,</span><br><span class="line">    <span class="string">'6'</span>: <span class="string">'r'</span>,</span><br><span class="line">    <span class="string">'f'</span>: <span class="string">'s'</span>,</span><br><span class="line">    <span class="string">'p'</span>: <span class="string">'t'</span>,</span><br><span class="line">    <span class="string">'7'</span>: <span class="string">'u'</span>,</span><br><span class="line">    <span class="string">'e'</span>: <span class="string">'v'</span>,</span><br><span class="line">    <span class="string">'o'</span>: <span class="string">'w'</span>,</span><br><span class="line">    <span class="string">'8'</span>: <span class="string">'1'</span>,</span><br><span class="line">    <span class="string">'d'</span>: <span class="string">'2'</span>,</span><br><span class="line">    <span class="string">'n'</span>: <span class="string">'3'</span>,</span><br><span class="line">    <span class="string">'9'</span>: <span class="string">'4'</span>,</span><br><span class="line">    <span class="string">'c'</span>: <span class="string">'5'</span>,</span><br><span class="line">    <span class="string">'m'</span>: <span class="string">'6'</span>,</span><br><span class="line">    <span class="string">'0'</span>: <span class="string">'7'</span>,</span><br><span class="line">    <span class="string">'b'</span>: <span class="string">'8'</span>,</span><br><span class="line">    <span class="string">'l'</span>: <span class="string">'9'</span>,</span><br><span class="line">    <span class="string">'a'</span>: <span class="string">'0'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># str 的translate方法需要用单个字符的十进制unicode编码作为key</span></span><br><span class="line"><span class="comment"># value 中的数字会被当成十进制unicode编码转换成字符</span></span><br><span class="line"><span class="comment"># 也可以直接用字符串作为value</span></span><br><span class="line">char_table = &#123;ord(key): ord(value) <span class="keyword">for</span> key, value <span class="keyword">in</span> char_table.items()&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">decode</span><span class="params">(url)</span>:</span></span><br><span class="line">    <span class="comment"># 先替换字符串</span></span><br><span class="line">    <span class="keyword">for</span> key, value <span class="keyword">in</span> str_table.items():</span><br><span class="line">        url = url.replace(key, value)</span><br><span class="line">    <span class="comment"># 再替换剩下的字符</span></span><br><span class="line">    <span class="keyword">return</span> url.translate(char_table)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    url = <span class="string">r"ippr_z2C$qAzdH3FAzdH3Ffl_z&amp;e3Bftgwt42_z&amp;e3BvgAzdH3F4omlaAzdH3Faa8W3ZyEpymRmx3Y1p7bb&amp;mla"</span></span><br><span class="line">    print(decode(url))</span><br></pre></td></tr></table></figure></p>
<p>测试成功！</p>
<p>再从JSON字符串中找出所有的originURL：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">re_url = re.compile(<span class="string">r'"objURL":"(.*?)"'</span>)</span><br><span class="line">imgs = re_url.findall(html)</span><br></pre></td></tr></table></figure></p>
<p>格式化JSON推荐使用Chrome的<a href="https://chrome.google.com/webstore/detail/iahnhfdhidomcpggpaimmmahffihkfnj" target="_blank" rel="external">JSON handle插件</a><br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9CJSON%20Handle.png" alt="JSON hanlde插件效果"></p>
<h2 id="单线程下载脚本">单线程下载脚本</h2><p>整理一下流程：</p>
<ol>
<li>生成网址列表</li>
<li>发送HTTP请求获取json数据</li>
<li>解析数据得到网址</li>
<li>下载</li>
</ol>
<p>整合一下上面的代码，可以写出单线程的下载脚本：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Author: loveNight</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> itertools</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">str_table = &#123;</span><br><span class="line">    <span class="string">'_z2C$q'</span>: <span class="string">':'</span>,</span><br><span class="line">    <span class="string">'_z&amp;e3B'</span>: <span class="string">'.'</span>,</span><br><span class="line">    <span class="string">'AzdH3F'</span>: <span class="string">'/'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">char_table = &#123;</span><br><span class="line">    <span class="string">'w'</span>: <span class="string">'a'</span>,</span><br><span class="line">    <span class="string">'k'</span>: <span class="string">'b'</span>,</span><br><span class="line">    <span class="string">'v'</span>: <span class="string">'c'</span>,</span><br><span class="line">    <span class="string">'1'</span>: <span class="string">'d'</span>,</span><br><span class="line">    <span class="string">'j'</span>: <span class="string">'e'</span>,</span><br><span class="line">    <span class="string">'u'</span>: <span class="string">'f'</span>,</span><br><span class="line">    <span class="string">'2'</span>: <span class="string">'g'</span>,</span><br><span class="line">    <span class="string">'i'</span>: <span class="string">'h'</span>,</span><br><span class="line">    <span class="string">'t'</span>: <span class="string">'i'</span>,</span><br><span class="line">    <span class="string">'3'</span>: <span class="string">'j'</span>,</span><br><span class="line">    <span class="string">'h'</span>: <span class="string">'k'</span>,</span><br><span class="line">    <span class="string">'s'</span>: <span class="string">'l'</span>,</span><br><span class="line">    <span class="string">'4'</span>: <span class="string">'m'</span>,</span><br><span class="line">    <span class="string">'g'</span>: <span class="string">'n'</span>,</span><br><span class="line">    <span class="string">'5'</span>: <span class="string">'o'</span>,</span><br><span class="line">    <span class="string">'r'</span>: <span class="string">'p'</span>,</span><br><span class="line">    <span class="string">'q'</span>: <span class="string">'q'</span>,</span><br><span class="line">    <span class="string">'6'</span>: <span class="string">'r'</span>,</span><br><span class="line">    <span class="string">'f'</span>: <span class="string">'s'</span>,</span><br><span class="line">    <span class="string">'p'</span>: <span class="string">'t'</span>,</span><br><span class="line">    <span class="string">'7'</span>: <span class="string">'u'</span>,</span><br><span class="line">    <span class="string">'e'</span>: <span class="string">'v'</span>,</span><br><span class="line">    <span class="string">'o'</span>: <span class="string">'w'</span>,</span><br><span class="line">    <span class="string">'8'</span>: <span class="string">'1'</span>,</span><br><span class="line">    <span class="string">'d'</span>: <span class="string">'2'</span>,</span><br><span class="line">    <span class="string">'n'</span>: <span class="string">'3'</span>,</span><br><span class="line">    <span class="string">'9'</span>: <span class="string">'4'</span>,</span><br><span class="line">    <span class="string">'c'</span>: <span class="string">'5'</span>,</span><br><span class="line">    <span class="string">'m'</span>: <span class="string">'6'</span>,</span><br><span class="line">    <span class="string">'0'</span>: <span class="string">'7'</span>,</span><br><span class="line">    <span class="string">'b'</span>: <span class="string">'8'</span>,</span><br><span class="line">    <span class="string">'l'</span>: <span class="string">'9'</span>,</span><br><span class="line">    <span class="string">'a'</span>: <span class="string">'0'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># str 的translate方法需要用单个字符的十进制unicode编码作为key</span></span><br><span class="line"><span class="comment"># value 中的数字会被当成十进制unicode编码转换成字符</span></span><br><span class="line"><span class="comment"># 也可以直接用字符串作为value</span></span><br><span class="line">char_table = &#123;ord(key): ord(value) <span class="keyword">for</span> key, value <span class="keyword">in</span> char_table.items()&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解码图片URL</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">decode</span><span class="params">(url)</span>:</span></span><br><span class="line">    <span class="comment"># 先替换字符串</span></span><br><span class="line">    <span class="keyword">for</span> key, value <span class="keyword">in</span> str_table.items():</span><br><span class="line">        url = url.replace(key, value)</span><br><span class="line">    <span class="comment"># 再替换剩下的字符</span></span><br><span class="line">    <span class="keyword">return</span> url.translate(char_table)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 生成网址列表</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">buildUrls</span><span class="params">(word)</span>:</span></span><br><span class="line">    word = urllib.parse.quote(word)</span><br><span class="line">    url = <span class="string">r"http://image.baidu.com/search/acjson?tn=resultjson_com&amp;ipn=rj&amp;ct=201326592&amp;fp=result&amp;queryWord=&#123;word&#125;&amp;cl=2&amp;lm=-1&amp;ie=utf-8&amp;oe=utf-8&amp;st=-1&amp;ic=0&amp;word=&#123;word&#125;&amp;face=0&amp;istype=2nc=1&amp;pn=&#123;pn&#125;&amp;rn=60"</span></span><br><span class="line">    urls = (url.format(word=word, pn=x) <span class="keyword">for</span> x <span class="keyword">in</span> itertools.count(start=<span class="number">0</span>, step=<span class="number">60</span>))</span><br><span class="line">    <span class="keyword">return</span> urls</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解析JSON获取图片URL</span></span><br><span class="line">re_url = re.compile(<span class="string">r'"objURL":"(.*?)"'</span>)</span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">resolveImgUrl</span><span class="params">(html)</span>:</span></span><br><span class="line">    imgUrls = [decode(x) <span class="keyword">for</span> x <span class="keyword">in</span> re_url.findall(html)]</span><br><span class="line">    <span class="keyword">return</span> imgUrls</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">downImg</span><span class="params">(imgUrl, dirpath, imgName)</span>:</span></span><br><span class="line">    filename = os.path.join(dirpath, imgName)</span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        res = requests.get(imgUrl, timeout=<span class="number">15</span>)</span><br><span class="line">        <span class="keyword">if</span> str(res.status_code)[<span class="number">0</span>] == <span class="string">"4"</span>:</span><br><span class="line">            print(str(res.status_code), <span class="string">":"</span> , imgUrl)</span><br><span class="line">            <span class="keyword">return</span> <span class="keyword">False</span></span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">        print(<span class="string">"抛出异常："</span>, imgUrl)</span><br><span class="line">        print(e)</span><br><span class="line">        <span class="keyword">return</span> <span class="keyword">False</span></span><br><span class="line">    <span class="keyword">with</span> open(filename, <span class="string">"wb"</span>) <span class="keyword">as</span> f:</span><br><span class="line">        f.write(res.content)</span><br><span class="line">    <span class="keyword">return</span> <span class="keyword">True</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">mkDir</span><span class="params">(dirName)</span>:</span></span><br><span class="line">    dirpath = os.path.join(sys.path[<span class="number">0</span>], dirName)</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(dirpath):</span><br><span class="line">        os.mkdir(dirpath)</span><br><span class="line">    <span class="keyword">return</span> dirpath</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    print(<span class="string">"欢迎使用百度图片下载脚本！\n目前仅支持单个关键词。"</span>)</span><br><span class="line">    print(<span class="string">"下载结果保存在脚本目录下的results文件夹中。"</span>)</span><br><span class="line">    print(<span class="string">"="</span> * <span class="number">50</span>)</span><br><span class="line">    word = input(<span class="string">"请输入你要下载的图片关键词：\n"</span>)</span><br><span class="line"></span><br><span class="line">    dirpath = mkDir(<span class="string">"results"</span>)</span><br><span class="line"></span><br><span class="line">    urls = buildUrls(word)</span><br><span class="line">    index = <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">        print(<span class="string">"正在请求："</span>, url)</span><br><span class="line">        html = requests.get(url, timeout=<span class="number">10</span>).content.decode(<span class="string">'utf-8'</span>)</span><br><span class="line">        imgUrls = resolveImgUrl(html)</span><br><span class="line">        <span class="keyword">if</span> len(imgUrls) == <span class="number">0</span>:  <span class="comment"># 没有图片则结束</span></span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line">        <span class="keyword">for</span> url <span class="keyword">in</span> imgUrls:</span><br><span class="line">            <span class="keyword">if</span> downImg(url, dirpath, str(index) + <span class="string">".jpg"</span>):</span><br><span class="line">                index += <span class="number">1</span></span><br><span class="line">                print(<span class="string">"已下载 %s 张"</span> % index)</span><br></pre></td></tr></table></figure></p>
<p>执行脚本：<br><img src="http://7xo8t2.com1.z0.glb.clouddn.com/img/%E7%BC%96%E7%A8%8B/Python/Python%203%20%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E6%90%9C%E7%B4%A2%E7%BB%93%E6%9E%9C%E5%8D%95%E7%BA%BF%E7%A8%8B%E4%B8%8B%E8%BD%BD%E7%BB%93%E6%9E%9C.png" alt="单线程下载结果"></p>
<p>查看同目录下的results文件夹，又看到了亲切的「他」。</p>
<h2 id="多线程下载脚本">多线程下载脚本</h2><p>上面的代码仍然有改进空间：</p>
<ol>
<li>从JSON数据看，该关键词相关的图片有一千多张，单线程下载太慢了，时间都花在网络和硬盘IO上。加上多线程可以大大提升效率。</li>
<li>既然1中已经获取到图片总数，那么网址的无限容量生成器可以改成list，方便添加多线程。</li>
</ol>
<p>多线程一直没学好，想不到更优雅的写法，大家将就看一下吧，欢迎提出改进建议。</p>
<p>百度图片下载脚本之多线程版：<br><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#!/usr/bin/env python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="comment"># @Author: loveNight</span></span><br><span class="line"><span class="comment"># @Date:   2015-10-28 19:59:24</span></span><br><span class="line"><span class="comment"># @Last Modified by:   loveNight</span></span><br><span class="line"><span class="comment"># @Last Modified time: 2015-11-15 19:24:57</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> threading</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime <span class="keyword">as</span> dt</span><br><span class="line"><span class="keyword">from</span> multiprocessing.dummy <span class="keyword">import</span> Pool</span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Queue</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">BaiduImgDownloader</span><span class="params">(object)</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="string">"""百度图片下载工具，目前只支持单个关键词"""</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 解码网址用的映射表</span></span><br><span class="line">    str_table = &#123;</span><br><span class="line">        <span class="string">'_z2C$q'</span>: <span class="string">':'</span>,</span><br><span class="line">        <span class="string">'_z&amp;e3B'</span>: <span class="string">'.'</span>,</span><br><span class="line">        <span class="string">'AzdH3F'</span>: <span class="string">'/'</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    char_table = &#123;</span><br><span class="line">        <span class="string">'w'</span>: <span class="string">'a'</span>,</span><br><span class="line">        <span class="string">'k'</span>: <span class="string">'b'</span>,</span><br><span class="line">        <span class="string">'v'</span>: <span class="string">'c'</span>,</span><br><span class="line">        <span class="string">'1'</span>: <span class="string">'d'</span>,</span><br><span class="line">        <span class="string">'j'</span>: <span class="string">'e'</span>,</span><br><span class="line">        <span class="string">'u'</span>: <span class="string">'f'</span>,</span><br><span class="line">        <span class="string">'2'</span>: <span class="string">'g'</span>,</span><br><span class="line">        <span class="string">'i'</span>: <span class="string">'h'</span>,</span><br><span class="line">        <span class="string">'t'</span>: <span class="string">'i'</span>,</span><br><span class="line">        <span class="string">'3'</span>: <span class="string">'j'</span>,</span><br><span class="line">        <span class="string">'h'</span>: <span class="string">'k'</span>,</span><br><span class="line">        <span class="string">'s'</span>: <span class="string">'l'</span>,</span><br><span class="line">        <span class="string">'4'</span>: <span class="string">'m'</span>,</span><br><span class="line">        <span class="string">'g'</span>: <span class="string">'n'</span>,</span><br><span class="line">        <span class="string">'5'</span>: <span class="string">'o'</span>,</span><br><span class="line">        <span class="string">'r'</span>: <span class="string">'p'</span>,</span><br><span class="line">        <span class="string">'q'</span>: <span class="string">'q'</span>,</span><br><span class="line">        <span class="string">'6'</span>: <span class="string">'r'</span>,</span><br><span class="line">        <span class="string">'f'</span>: <span class="string">'s'</span>,</span><br><span class="line">        <span class="string">'p'</span>: <span class="string">'t'</span>,</span><br><span class="line">        <span class="string">'7'</span>: <span class="string">'u'</span>,</span><br><span class="line">        <span class="string">'e'</span>: <span class="string">'v'</span>,</span><br><span class="line">        <span class="string">'o'</span>: <span class="string">'w'</span>,</span><br><span class="line">        <span class="string">'8'</span>: <span class="string">'1'</span>,</span><br><span class="line">        <span class="string">'d'</span>: <span class="string">'2'</span>,</span><br><span class="line">        <span class="string">'n'</span>: <span class="string">'3'</span>,</span><br><span class="line">        <span class="string">'9'</span>: <span class="string">'4'</span>,</span><br><span class="line">        <span class="string">'c'</span>: <span class="string">'5'</span>,</span><br><span class="line">        <span class="string">'m'</span>: <span class="string">'6'</span>,</span><br><span class="line">        <span class="string">'0'</span>: <span class="string">'7'</span>,</span><br><span class="line">        <span class="string">'b'</span>: <span class="string">'8'</span>,</span><br><span class="line">        <span class="string">'l'</span>: <span class="string">'9'</span>,</span><br><span class="line">        <span class="string">'a'</span>: <span class="string">'0'</span></span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    re_objURL = re.compile(<span class="string">r'"objURL":"(.*?)".*?"type":"(.*?)"'</span>)</span><br><span class="line">    re_downNum = re.compile(<span class="string">r"已下载\s(\d+)\s张图片"</span>)</span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36"</span>,</span><br><span class="line">        <span class="string">"Accept-Encoding"</span>: <span class="string">"gzip, deflate, sdch"</span>,</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, word, dirpath=None, processNum=<span class="number">30</span>)</span>:</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">" "</span> <span class="keyword">in</span> word:</span><br><span class="line">            <span class="keyword">raise</span> AttributeError(<span class="string">"本脚本仅支持单个关键字"</span>)</span><br><span class="line">        self.word = word</span><br><span class="line">        self.char_table = &#123;ord(key): ord(value)</span><br><span class="line">                           <span class="keyword">for</span> key, value <span class="keyword">in</span> BaiduImgDownloader.char_table.items()&#125;</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> dirpath:</span><br><span class="line">            dirpath = os.path.join(sys.path[<span class="number">0</span>], <span class="string">'results'</span>)</span><br><span class="line">        self.dirpath = dirpath</span><br><span class="line">        self.jsonUrlFile = os.path.join(sys.path[<span class="number">0</span>], <span class="string">'jsonUrl.txt'</span>)</span><br><span class="line">        self.logFile = os.path.join(sys.path[<span class="number">0</span>], <span class="string">'logInfo.txt'</span>)</span><br><span class="line">        self.errorFile = os.path.join(sys.path[<span class="number">0</span>], <span class="string">'errorUrl.txt'</span>)</span><br><span class="line">        <span class="keyword">if</span> os.path.exists(self.errorFile):</span><br><span class="line">            os.remove(self.errorFile)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(self.dirpath):</span><br><span class="line">            os.mkdir(self.dirpath)</span><br><span class="line">        self.pool = Pool(<span class="number">30</span>)</span><br><span class="line">        self.session = requests.Session()</span><br><span class="line">        self.session.headers = BaiduImgDownloader.headers</span><br><span class="line">        self.queue = Queue()</span><br><span class="line">        self.messageQueue = Queue()</span><br><span class="line">        self.index = <span class="number">0</span> <span class="comment"># 图片起始编号，牵涉到计数，不要更改</span></span><br><span class="line">        self.promptNum = <span class="number">10</span> <span class="comment"># 下载几张图片提示一次</span></span><br><span class="line">        self.lock = threading.Lock()</span><br><span class="line">        self.delay = <span class="number">1.5</span>  <span class="comment"># 网络请求太频繁会被封</span></span><br><span class="line">        self.QUIT = <span class="string">"QUIT"</span>  <span class="comment"># Queue中表示任务结束</span></span><br><span class="line">        self.printPrefix = <span class="string">"**"</span> <span class="comment"># 用于指定在控制台输出</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="comment"># 控制台输出线程</span></span><br><span class="line">        t = threading.Thread(target=self.__log)</span><br><span class="line">        t.setDaemon(<span class="keyword">True</span>)</span><br><span class="line">        t.start()</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"脚本开始执行"</span>)</span><br><span class="line">        start_time = dt.now()</span><br><span class="line">        urls = self.__buildUrls()</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"已获取 %s 个Json请求网址"</span> % len(urls))</span><br><span class="line">        <span class="comment"># 解析出所有图片网址，该方法会阻塞直到任务完成</span></span><br><span class="line">        self.pool.map(self.__resolveImgUrl, urls)</span><br><span class="line">        <span class="keyword">while</span> self.queue.qsize():</span><br><span class="line">            imgs = self.queue.get()</span><br><span class="line">            self.pool.map_async(self.__downImg, imgs)</span><br><span class="line">        self.pool.close()</span><br><span class="line">        self.pool.join()</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"下载完成！已下载 %s 张图片，总用时 %s"</span> %</span><br><span class="line">                              (self.index, dt.now() - start_time))</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"请到 %s 查看结果！"</span> % self.dirpath)</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"错误信息保存在 %s"</span> % self.errorFile)</span><br><span class="line">        self.messageQueue.put(self.QUIT)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__log</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""控制台输出，加锁以免被多线程打乱"""</span></span><br><span class="line">        <span class="keyword">with</span> open(self.logFile, <span class="string">"w"</span>, encoding = <span class="string">"utf-8"</span>) <span class="keyword">as</span> f:</span><br><span class="line">            <span class="keyword">while</span> <span class="keyword">True</span>:</span><br><span class="line">                message = self.messageQueue.get()</span><br><span class="line">                <span class="keyword">if</span> message == self.QUIT:</span><br><span class="line">                    <span class="keyword">break</span></span><br><span class="line">                message = str(dt.now()) + <span class="string">" "</span> + message</span><br><span class="line">                <span class="keyword">if</span> self.printPrefix  <span class="keyword">in</span> message:</span><br><span class="line">                    print(message)</span><br><span class="line">                <span class="keyword">elif</span> <span class="string">"已下载"</span> <span class="keyword">in</span> message:</span><br><span class="line">                    <span class="comment"># 下载N张图片提示一次</span></span><br><span class="line">                    downNum = self.re_downNum.findall(message)</span><br><span class="line">                    <span class="keyword">if</span> downNum <span class="keyword">and</span> int(downNum[<span class="number">0</span>]) % self.promptNum == <span class="number">0</span>:</span><br><span class="line">                        print(message)</span><br><span class="line">                f.write(message + <span class="string">'\n'</span>)</span><br><span class="line">                f.flush()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__getIndex</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""获取文件编号"""</span></span><br><span class="line">        self.lock.acquire()</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="keyword">return</span> self.index</span><br><span class="line">        <span class="keyword">finally</span>:</span><br><span class="line">            self.index += <span class="number">1</span></span><br><span class="line">            self.lock.release()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">decode</span><span class="params">(self, url)</span>:</span></span><br><span class="line">        <span class="string">"""解码图片URL</span><br><span class="line">        解码前：</span><br><span class="line">        ippr_z2C$qAzdH3FAzdH3Ffl_z&amp;e3Bftgwt42_z&amp;e3BvgAzdH3F4omlaAzdH3Faa8W3ZyEpymRmx3Y1p7bb&amp;mla</span><br><span class="line">        解码后：</span><br><span class="line">        http://s9.sinaimg.cn/mw690/001WjZyEty6R6xjYdtu88&amp;690</span><br><span class="line">        """</span></span><br><span class="line">        <span class="comment"># 先替换字符串</span></span><br><span class="line">        <span class="keyword">for</span> key, value <span class="keyword">in</span> self.str_table.items():</span><br><span class="line">            url = url.replace(key, value)</span><br><span class="line">        <span class="comment"># 再替换剩下的字符</span></span><br><span class="line">        <span class="keyword">return</span> url.translate(self.char_table)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__buildUrls</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="string">"""json请求网址生成器"""</span></span><br><span class="line">        word = urllib.parse.quote(self.word)</span><br><span class="line">        url = <span class="string">r"http://image.baidu.com/search/acjson?tn=resultjson_com&amp;ipn=rj&amp;ct=201326592&amp;fp=result&amp;queryWord=&#123;word&#125;&amp;cl=2&amp;lm=-1&amp;ie=utf-8&amp;oe=utf-8&amp;st=-1&amp;ic=0&amp;word=&#123;word&#125;&amp;face=0&amp;istype=2nc=1&amp;pn=&#123;pn&#125;&amp;rn=60"</span></span><br><span class="line">        time.sleep(self.delay)</span><br><span class="line">        html = self.session.get(url.format(word=word, pn=<span class="number">0</span>), timeout = <span class="number">15</span>).content.decode(<span class="string">'utf-8'</span>)</span><br><span class="line">        results = re.findall(<span class="string">r'"displayNum":(\d+),'</span>, html)</span><br><span class="line">        maxNum = int(results[<span class="number">0</span>]) <span class="keyword">if</span> results <span class="keyword">else</span> <span class="number">0</span></span><br><span class="line">        urls = [url.format(word=word, pn=x)</span><br><span class="line">                <span class="keyword">for</span> x <span class="keyword">in</span> range(<span class="number">0</span>, maxNum + <span class="number">1</span>, <span class="number">60</span>)]</span><br><span class="line">        <span class="keyword">with</span> open(self.jsonUrlFile, <span class="string">"w"</span>, encoding=<span class="string">"utf-8"</span>) <span class="keyword">as</span> f:</span><br><span class="line">            <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">                f.write(url + <span class="string">"\n"</span>)</span><br><span class="line">        <span class="keyword">return</span> urls</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__resolveImgUrl</span><span class="params">(self, url)</span>:</span></span><br><span class="line">        <span class="string">"""从指定网页中解析出图片URL"""</span></span><br><span class="line">        time.sleep(self.delay)</span><br><span class="line">        html = self.session.get(url, timeout = <span class="number">15</span>).content.decode(<span class="string">'utf-8'</span>)</span><br><span class="line">        datas = self.re_objURL.findall(html)</span><br><span class="line">        imgs = [Image(self.decode(x[<span class="number">0</span>]), x[<span class="number">1</span>]) <span class="keyword">for</span> x <span class="keyword">in</span> datas]</span><br><span class="line">        self.messageQueue.put(self.printPrefix + <span class="string">"已解析出 %s 个图片网址"</span> % len(imgs))</span><br><span class="line">        self.queue.put(imgs)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__downImg</span><span class="params">(self, img)</span>:</span></span><br><span class="line">        <span class="string">"""下载单张图片，传入的是Image对象"""</span></span><br><span class="line">        imgUrl = img.url</span><br><span class="line">        <span class="comment"># self.messageQueue.put("线程 %s 正在下载 %s " %</span></span><br><span class="line">        <span class="comment">#          (threading.current_thread().name, imgUrl))</span></span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            time.sleep(self.delay)</span><br><span class="line">            res = self.session.get(imgUrl, timeout = <span class="number">15</span>)</span><br><span class="line">            message = <span class="keyword">None</span></span><br><span class="line">            <span class="keyword">if</span> str(res.status_code)[<span class="number">0</span>] == <span class="string">"4"</span>:</span><br><span class="line">                message = <span class="string">"\n%s： %s"</span> % (res.status_code, imgUrl)</span><br><span class="line">            <span class="keyword">elif</span> <span class="string">"text/html"</span> <span class="keyword">in</span> res.headers[<span class="string">"Content-Type"</span>]:</span><br><span class="line">                message = <span class="string">"\n无法打开图片： %s"</span> % imgUrl</span><br><span class="line">        <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">            message = <span class="string">"\n抛出异常： %s\n%s"</span> % (imgUrl, str(e))</span><br><span class="line">        <span class="keyword">finally</span>:</span><br><span class="line">            <span class="keyword">if</span> message:</span><br><span class="line">                self.messageQueue.put(message)</span><br><span class="line">                self.__saveError(message)</span><br><span class="line">                <span class="keyword">return</span></span><br><span class="line">        index = self.__getIndex()</span><br><span class="line">        <span class="comment"># index从0开始</span></span><br><span class="line">        self.messageQueue.put(<span class="string">"已下载 %s 张图片：%s"</span> % (index + <span class="number">1</span>, imgUrl))</span><br><span class="line">        filename = os.path.join(self.dirpath, str(index) + <span class="string">"."</span> + img.type)</span><br><span class="line">        <span class="keyword">with</span> open(filename, <span class="string">"wb"</span>) <span class="keyword">as</span> f:</span><br><span class="line">            f.write(res.content)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__saveError</span><span class="params">(self, message)</span>:</span></span><br><span class="line">        self.lock.acquire()</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="keyword">with</span> open(self.errorFile, <span class="string">"a"</span>, encoding=<span class="string">"utf-8"</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(message)</span><br><span class="line">        <span class="keyword">finally</span>:</span><br><span class="line">            self.lock.release()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Image</span><span class="params">(object)</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="string">"""图片类，保存图片信息"""</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, url, type)</span>:</span></span><br><span class="line">        super(Image, self).__init__()</span><br><span class="line">        self.url = url</span><br><span class="line">        self.type = type</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    print(<span class="string">"欢迎使用百度图片下载脚本！\n目前仅支持单个关键词。"</span>)</span><br><span class="line">    print(<span class="string">"下载结果保存在脚本目录下的results文件夹中。"</span>)</span><br><span class="line">    print(<span class="string">"="</span> * <span class="number">50</span>)</span><br><span class="line">    word = input(<span class="string">"请输入你要下载的图片关键词：\n"</span>)</span><br><span class="line">    down = BaiduImgDownloader(word)</span><br><span class="line">    down.start()</span><br></pre></td></tr></table></figure></p>
<p>执行脚本：</p>
<pre><code>欢迎使用百度图片下载脚本！
目前仅支持单个关键词。
下载结果保存在脚本目录下的results文件夹中。
==================================================
请输入你要下载的图片关键词：
长者蛤
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">11.726878</span> **脚本开始执行
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">16.292022</span> **已获取 <span class="number">20</span> 个Json请求网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.885767</span> **已解析出 <span class="number">30</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.917020</span> **已解析出 <span class="number">60</span> 个图片网址
.....
.....中间省略
.....
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">31.726739</span> 已下载 <span class="number">980</span> 张图片：http:<span class="comment">//bbs.nantaihu.com/bbs/UpImages2008/2010/8/3/U97946_201083171218512-2.jpg</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">32.695518</span> 已下载 <span class="number">990</span> 张图片：http:<span class="comment">//pf.u.51img1.com/f9/7f/huangbaoling0_180.gif?v=20120224161006</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">45.473957</span> 已下载 <span class="number">1000</span> 张图片：http:<span class="comment">//library.taiwanschoolnet.org/cyberfair2003/C0312970394/narrative.files/image018.jpg</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">50.749544</span> **下载完成！已下载 <span class="number">1000</span> 张图片，总用时 <span class="number">0</span>:<span class="number">08</span>:<span class="number">39.022666</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">50.765169</span> **请到 F:\PythonWorkspace\Learn\results 查看结果！
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">33</span>:<span class="number">50.858880</span> **错误信息保存在 F:\PythonWorkspace\Learn\errorUrl.txt
</code></pre><p>合租室友太多，平时网速很慢，经常连百度都打不开，要刷新多次。下午测试的时候可以2分钟下完，现在居然要近9分钟。</p>
<p><code>errorUrl.txt</code>的内容</p>
<pre><code>无法打开图片： http:<span class="comment">//s3.t.itc.cn/mblog/pic/20136_20_14/f_b3a1be1796819229739.png</span>
<span class="number">403</span>： http:<span class="comment">//www.longhoo.net/gb/longhoo/culture/view/images/00024483.jpg</span>
无法打开图片： http:<span class="comment">//bhyjk.cn/images/IMG44zj10.jpg</span>
<span class="number">404</span>： http:<span class="comment">//ebook.indaa.com.cn/upload/baike/images/2960001-2970000/2968900/adee30ddd41190838c10291d.jpg</span>
抛出异常： http:<span class="comment">//www.jzxzj.com/images/xs.jpg</span>
<span class="function"><span class="title">HTTPConnectionPool</span><span class="params">(host=<span class="string">'www.jzxzj.com'</span>, port=<span class="number">80</span>)</span></span>: Max retries exceeded with url: /images/xs<span class="class">.jpg</span> (Caused by <span class="function"><span class="title">NewConnectionError</span><span class="params">(<span class="string">'&lt;requests.packages.urllib3.connection.HTTPConnection object at 0x0851FBB0&gt;: Failed to establish a new connection: [Errno 11002] getaddrinfo failed'</span>,)</span></span>)
抛出异常： http:<span class="comment">//static.acfun.mm111.net/h/image/20147/846b7553851b6edb2305c6b87f4aed71.jpg</span>
<span class="function"><span class="title">HTTPConnectionPool</span><span class="params">(host=<span class="string">'static.acfun.mm111.net'</span>, port=<span class="number">80</span>)</span></span>: Max retries exceeded with url: /h/image/<span class="number">20147</span>/<span class="number">846</span>b7553851b6edb2305c6b87f4aed71<span class="class">.jpg</span> (Caused by <span class="function"><span class="title">NewConnectionError</span><span class="params">(<span class="string">'&lt;requests.packages.urllib3.connection.HTTPConnection object at 0x0888B710&gt;: Failed to establish a new connection: [Errno 11001] getaddrinfo failed'</span>,)</span></span>)
抛出异常： http:<span class="comment">//static.acfun.mm111.net/h/image/2015-0-4/23ebeaf5-5130-4072-b90e-48ce4d122ac8.jpg</span>
<span class="function"><span class="title">HTTPConnectionPool</span><span class="params">(host=<span class="string">'static.acfun.mm111.net'</span>, port=<span class="number">80</span>)</span></span>: Max retries exceeded with url: /h/image/<span class="number">2015</span>-<span class="number">0</span>-<span class="number">4</span>/<span class="number">23</span>ebeaf5-<span class="number">5130</span>-<span class="number">4072</span>-b90e-<span class="number">48</span>ce4d122ac8<span class="class">.jpg</span> (Caused by <span class="function"><span class="title">NewConnectionError</span><span class="params">(<span class="string">'&lt;requests.packages.urllib3.connection.HTTPConnection object at 0x0888F1F0&gt;: Failed to establish a new connection: [Errno 11001] getaddrinfo failed'</span>,)</span></span>)
<span class="number">404</span>： http:<span class="comment">//www.zipooo.com/tupian/1/%CE%EF%C1%F7%BF%EC%B5%DD.gif</span>
无法打开图片： http:<span class="comment">//www.zhiyin.cn/hsbk/fakeimg/7939883.jpeg</span>
无法打开图片： http:<span class="comment">//www.zhiyin.cn/hsbk/fakeimg/21753396.jpeg</span>
.....
</code></pre><p><code>logInfo.txt</code>内容：</p>
<pre><code><span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">11.726878</span> **脚本开始执行
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">16.292022</span> **已获取 <span class="number">20</span> 个Json请求网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.885767</span> **已解析出 <span class="number">30</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.917020</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.948262</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.963895</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">17.995146</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.010769</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.057638</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.057638</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.073264</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.073264</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.088890</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.088890</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.104516</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.104516</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.104516</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.104516</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.245137</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.260764</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.260764</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">18.479555</span> **已解析出 <span class="number">60</span> 个图片网址
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">27.532965</span> 已下载 <span class="number">1</span> 张图片：http:<span class="comment">//imgsrc.baidu.com/baike/abpic/item/43e6c7335b379350ad4b5f92.jpg</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">29.084964</span>
无法打开图片： http:<span class="comment">//s3.t.itc.cn/mblog/pic/20136_20_14/f_b3a1be1796819229739.png</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">29.522473</span> 已下载 <span class="number">2</span> 张图片：http:<span class="comment">//p4.qhimg.com/dr/200_200_/t0196ad8cefbc7a2c7c.jpg</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">29.822650</span> 已下载 <span class="number">3</span> 张图片：http:<span class="comment">//library.taiwanschoolnet.org/cyberfair2003/C0312970394/narrative.files/image016.jpg</span>
<span class="number">2015</span>-<span class="number">11</span>-<span class="number">15</span> <span class="number">19</span>:<span class="number">25</span>:<span class="number">29.938687</span> 已下载 <span class="number">4</span> 张图片：http:<span class="comment">//blog.ylib.com/public/Utilities/ImageReader.aspx?/0/6/06c67eaf-09db-40a8-a117-c577a6d05d08.jpg</span>
....
</code></pre><p>仍然可以改进的地方：</p>
<ol>
<li>如果要搜索多个关键词，buildUrls方法应该怎么改？</li>
<li>如果脚本中途意外结束（比如被熊孩子点了X），如何继续下载？</li>
<li>线程池中的线程数需要多次测试才能找到最优值。</li>
</ol>

      
    </div>

    <div>
      
        
<div id="wechat_subscriber" style="display: block; padding: 10px 0; margin: 20px auto; width: 100%; text-align: center">
    <img id="wechat_subscriber_qcode" src="/wechat-qrcode.jpg" alt="loveNight wechat" style="width: 200px; max-width: 100%;"/>
    <div>我的微信公众号，放一些有趣的内容，不定期更新。</div>
</div>


      
    </div>

    <div>
      
        
  <div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
    <div></div>
    <button id="rewardButton" disable="enable" onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
      <span>赏</span>
    </button>
    <div id="QR" style="display: none;">
      
        <div id="wechat" style="display: inline-block">
          <img id="wechat_qr" src="/wechat-pay-qrcode.jpg" alt="loveNight WeChat Pay"/>
          <p>微信打赏</p>
        </div>
      
      
        <div id="alipay" style="display: inline-block">
          <img id="alipay_qr" src="/alipay-qrcode.jpg" alt="loveNight Alipay"/>
          <p>支付宝打赏</p>
        </div>
      
    </div>
  </div>


      
    </div>

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Python/" rel="tag">#Python</a>
          
            <a href="/tags/多线程/" rel="tag">#多线程</a>
          
        </div>
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2015/11/14/Python-3-json与正则式解析速度测试/" rel="next" title="Python 3 json与正则式解析速度测试">
                <i class="fa fa-chevron-left"></i> Python 3 json与正则式解析速度测试
              </a>
            
          </div>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2015/11/16/Python-3-安装第三方库时出现编译错误/" rel="prev" title="Python 3 安装第三方库报错Unable to find vcvarsall.bat">
                Python 3 安装第三方库报错Unable to find vcvarsall.bat <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
        <!-- JiaThis Button BEGIN -->
<div class="jiathis_style">
  <a class="jiathis_button_tsina"></a>
  <a class="jiathis_button_tqq"></a>
  <a class="jiathis_button_weixin"></a>
  <a class="jiathis_button_cqq"></a>
  <a class="jiathis_button_douban"></a>
  <a class="jiathis_button_renren"></a>
  <a class="jiathis_button_qzone"></a>
  <a class="jiathis_button_kaixin001"></a>
  <a class="jiathis_button_copy"></a>
  <a href="http://www.jiathis.com/share" class="jiathis jiathis_txt jiathis_separator jtico jtico_jiathis" target="_blank"></a>
  <a class="jiathis_counter_style"></a>
</div>
<script type="text/javascript" >
  var jiathis_config={
    hideMore:false
  }
</script>
<script type="text/javascript" src="http://v3.jiathis.com/code/jia.js" charset="utf-8"></script>
<!-- JiaThis Button END -->

      
    </div>
  </div>


          </div>
          


          
  <div class="comments" id="comments">
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel ">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/avatar.jpg"
               alt="loveNight" />
          <p class="site-author-name" itemprop="name">loveNight</p>
          <p class="site-description motion-element" itemprop="description">自古求真皆寂寞，唯挑心灯伴夜霭</p>
        </div>
        <nav class="site-state motion-element">
          <div class="site-state-item site-state-posts">
            <a href="/archives">
              <span class="site-state-item-count">46</span>
              <span class="site-state-item-name">日志</span>
            </a>
          </div>

          
            <div class="site-state-item site-state-categories">
              <a href="/categories">
                <span class="site-state-item-count">24</span>
                <span class="site-state-item-name">分类</span>
              </a>
            </div>
          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">68</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        
          <div class="feed-link motion-element">
            <a href="/atom.xml" rel="alternate">
              <i class="fa fa-rss"></i>
              RSS
            </a>
          </div>
        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/loveNight" target="_blank" title="Github">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Github
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://weibo.com/p/1005051069913250" target="_blank" title="Weibo">
                  
                    <i class="fa fa-fw fa-weibo"></i>
                  
                  Weibo
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="mailto:286242151@qq.com" target="_blank" title="Email">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Email
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://segmentfault.com/u/xuxian" target="_blank" title="SegmentFault">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  SegmentFault
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://www.zhihu.com/people/wu-wu-4-90" target="_blank" title="Zhihu">
                  
                    <i class="fa fa-fw fa-globe"></i>
                  
                  Zhihu
                </a>
              </span>
            
          
        </div>

        
        

        
        
          <div class="links-of-blogroll motion-element links-of-blogroll-inline">
            <div class="links-of-blogroll-title">
              <i class="fa  fa-fw fa-globe"></i>
              友情链接
            </div>
            <ul class="links-of-blogroll-list">
              
                <li class="links-of-blogroll-item">
                  <a href="http://blog.csdn.net/kinglearnjava" title="我的CSDN博客" target="_blank">我的CSDN博客</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://blog.sina.com.cn/u/1069913250" title="我的新浪博客" target="_blank">我的新浪博客</a>
                </li>
              
                <li class="links-of-blogroll-item">
                  <a href="http://ta.qq.com" title="腾讯分析" target="_blank">腾讯分析</a>
                </li>
              
            </ul>
          </div>
        

      </section>

      
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">
            
              
            
            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#下载简单页面"><span class="nav-number">1.</span> <span class="nav-text">下载简单页面</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#加载更多图片"><span class="nav-number">2.</span> <span class="nav-text">加载更多图片</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#单线程下载脚本"><span class="nav-number">3.</span> <span class="nav-text">单线程下载脚本</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#多线程下载脚本"><span class="nav-number">4.</span> <span class="nav-text">多线程下载脚本</span></a></li></ol></div>
            
          </div>
        </section>
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy;  2015 - 
  <span itemprop="copyrightYear">2017</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">loveNight


  &nbsp;&nbsp;|&nbsp;&nbsp;<span><a href="/sitemap.xml">Google网站地图</a></span>
  &nbsp;&nbsp;|&nbsp;&nbsp;<span><a href="/baidusitemap.xml">百度网站地图</a></span>

  </span>
</div>

<div class="powered-by">
  powered by <a class="theme-link" href="https://hexo.io">Hexo</a>
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Pisces
  </a>
</div>


&nbsp;&nbsp;|&nbsp;&nbsp;本站总点击 <span id="busuanzi_value_site_pv"></span> 次
&nbsp;&nbsp;|&nbsp;&nbsp;您是第 <span id="busuanzi_value_site_uv"></span> 位访客

<script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js">
</script>

        

        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  



  
  <script type="text/javascript" src="/vendors/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/vendors/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/vendors/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/vendors/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/vendors/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/vendors/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.0.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.0.1"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.0.1"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.0.1"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.0.1"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.0.1"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.0.1"></script>



  



  




  
  
  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length == 0) {
       search_path = "search.xml";
    }
    var path = "/" + search_path;
    // monitor main search box;

    function proceedsearch() {
      $("body").append('<div class="popoverlay">').css('overflow', 'hidden');
      $('.popup').toggle();

    }
    // search function;
    var searchFunc = function(path, search_id, content_id) {
    'use strict';
    $.ajax({
        url: path,
        dataType: "xml",
        async: true,
        success: function( xmlResponse ) {
            // get the contents from search data
            isfetched = true;
            $('.popup').detach().appendTo('.header-inner');
            var datas = $( "entry", xmlResponse ).map(function() {
                return {
                    title: $( "title", this ).text(),
                    content: $("content",this).text(),
                    url: $( "url" , this).text()
                };
            }).get();
            var $input = document.getElementById(search_id);
            var $resultContent = document.getElementById(content_id);
            $input.addEventListener('input', function(){
                var matchcounts = 0;
                var str='<ul class=\"search-result-list\">';
                var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                $resultContent.innerHTML = "";
                if (this.value.trim().length > 1) {
                // perform local searching
                datas.forEach(function(data) {
                    var isMatch = true;
                    var content_index = [];
                    var data_title = data.title.trim().toLowerCase();
                    var data_content = data.content.trim().replace(/<[^>]+>/g,"").toLowerCase();
                    var data_url = data.url;
                    var index_title = -1;
                    var index_content = -1;
                    var first_occur = -1;
                    // only match artiles with not empty titles and contents
                    if(data_title != '' && data_content != '') {
                        keywords.forEach(function(keyword, i) {
                            index_title = data_title.indexOf(keyword);
                            index_content = data_content.indexOf(keyword);
                            if( index_title < 0 && index_content < 0 ){
                                isMatch = false;
                            } else {
                                if (index_content < 0) {
                                    index_content = 0;
                                }
                                if (i == 0) {
                                    first_occur = index_content;
                                }
                            }
                        });
                    }
                    // show search results
                    if (isMatch) {
                        matchcounts += 1;
                        str += "<li><a href='"+ data_url +"' class='search-result-title'>"+ data_title +"</a>";
                        var content = data.content.trim().replace(/<[^>]+>/g,"");
                        if (first_occur >= 0) {
                            // cut out 100 characters
                            var start = first_occur - 20;
                            var end = first_occur + 80;
                            if(start < 0){
                                start = 0;
                            }
                            if(start == 0){
                                end = 50;
                            }
                            if(end > content.length){
                                end = content.length;
                            }
                            var match_content = content.substring(start, end);
                            // highlight all keywords
                            keywords.forEach(function(keyword){
                                var regS = new RegExp(keyword, "gi");
                                match_content = match_content.replace(regS, "<b class=\"search-keyword\">"+keyword+"</b>");
                            });

                            str += "<p class=\"search-result\">" + match_content +"...</p>"
                        }
                        str += "</li>";
                    }
                })};
                str += "</ul>";
                if (matchcounts == 0) { str = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>' }
                if (keywords == "") { str = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>' }
                $resultContent.innerHTML = str;
            });
            proceedsearch();
        }
    });}

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched == false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };

    });

    $('.popup-btn-close').click(function(e){
      $('.popup').hide();
      $(".popoverlay").remove();
      $('body').css('overflow', '');
    });
    $('.popup').click(function(e){
      e.stopPropagation();
    });
  </script>


  

  

  

</body>
</html>
